package com.hdb.servlet;

import java.io.IOException;
import java.io.InputStream;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.Timer;
import java.util.TimerTask;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.hibernate.Query;
import org.hibernate.Session;

import com.hdb.downloder.HttpClientDownloader;
import com.hdb.pipeline.TestFilePipeline;
import com.hdb.pojo.NewsCss;
import com.hdb.processor.ProcessorTest;
import com.hdb.scheduler.SpikeFileCacheQueueScheduler;
import com.hdb.scheduler.SpikeRedisScheduler;
import com.hdb.util.HibernateUtil;
import com.hdb.util.UrlStringUtils;

import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.FilePipeline;

/**
 * Servlet implementation class MyServlet
 */
public class TestServlet extends HttpServlet {
	private static final long serialVersionUID = 1L;

	/**
	 * @see HttpServlet#HttpServlet()
	 */
	public TestServlet() {
		super();
		
	}

	/**
	 * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse
	 *      response)
	 */
	protected void doGet(HttpServletRequest request, final HttpServletResponse response)
			throws ServletException, IOException {
		response.setContentType("text/html;charset=utf-8");
		// 获取数据库数据
		Session session = HibernateUtil.getSession();
		List<NewsCss> list = null;
		try {
			Query query = session.createQuery("from NewsCss");
			list = query.list();
		} catch (Exception e) {
			e.printStackTrace();

			System.out.println("查询失败!");

		} finally {
			HibernateUtil.closeSession();
		}
		final List<NewsCss> listt=list;
		//获得源地址的集合
		final List<String> urlList = UrlStringUtils.getUrlList(listt);
		//读取配置文件
		Properties properties = new Properties();
		InputStream in = TestServlet.class.getClassLoader().getResourceAsStream("data.properties");
		properties.load(in);
		final String location = properties.getProperty("location");
		in.close();
		//定时任务
		new Timer().schedule(new TimerTask() {
			@Override
			public void run() {
				//需要定时的代码
				long startTime, endTime;
				startTime = System.currentTimeMillis();
				System.out.println("开始爬取...");
				int count=0;//记录条数
				
				for (NewsCss newsCss : listt) {
					//设置增量爬取的Scheduler
					SpikeFileCacheQueueScheduler fileUrl=new SpikeFileCacheQueueScheduler("D:\\webmagic\\url");
					fileUrl.setURLList(urlList);
					System.out.println(newsCss.getResource());
					ProcessorTest ProcessorTest=new ProcessorTest(newsCss);
					
					TestFilePipeline testFilePipeline= new TestFilePipeline(location);
					/*Spider.create(ProcessorTest).setScheduler(fileUrl).setDownloader(new HttpClientDownloader()).addUrl(newsCss.getResource())
							.addPipeline(new TestFilePipeline(location)).thread(5).run();//文件存储增量查询
*/					//设置增量爬取的redisScheduler
					/*
					 * 设置redis连接池
					 */
					JedisPoolConfig config = new JedisPoolConfig(); 
			        config.setMaxIdle(5); 
			        config.setMaxWaitMillis(10000); 
			        config.setTestOnBorrow(false); 
			        JedisPool jedisPool = new JedisPool(config,"127.0.0.1",6379);
			        
			        SpikeRedisScheduler redisScheduler=new SpikeRedisScheduler(jedisPool);
			        redisScheduler.setuRLList(urlList);
					Spider.create(ProcessorTest).setDownloader(new HttpClientDownloader()).addUrl(newsCss.getResource()).addPipeline(new ConsolePipeline()).addPipeline(new FilePipeline()).thread(5).run();//redis存储增量查询
			       
					count+= testFilePipeline.getCount();//count为每次爬取存储的记录数
				}
				endTime = System.currentTimeMillis();
				System.out.println("爬取结束，耗时约" + ((endTime - startTime) / 1000) + "秒，抓取了" +count + "条记录");
				
				try {
					response.getWriter().write("爬取完成！！！");
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
		},new Date(),1000*60*10);
		
	}

	/**
	 * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse
	 *      response)
	 */
	protected void doPost(HttpServletRequest request, HttpServletResponse response)
			throws ServletException, IOException {
		// TODO Auto-generated method stub
		doGet(request, response);
	}

}
